/*
+-----------------------------------------------------------------------------------------------------------------------
| Author: xank <xank@qq.com>  Blog：https://www.xank.cn
+-----------------------------------------------------------------------------------------------------------------------
| Pipeline 数据输出管道
|
*/

function isPlainObject(obj) {
  return Object.prototype.toString.call(obj) === '[object Object]'
}

function etlData(data) {
  for (let k in data) {
    let value = data[k]

    if (typeof value === 'string') {
      data[k] = value.replace(/[\r\n\t]/gm, '')
    } else if (isPlainObject(value)) {
      value = etlData(value)
    }
  }
}

function pipeline(NScan) {
  NScan.prototype.initPipeline = function() {
    const { insert } = this.options
    this.insert = insert
  }

  NScan.prototype.insertDb = async function(data) {
    const { url } = data

    // 数据清洗
    etlData(data)
    await this.insert(url, data)
  }
}

module.exports = pipeline
